// Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
// Project developers.  See the top-level LICENSE file for dates and other
// details.  No copyright assignment is required to contribute to VisIt.

// ************************************************************************* //
// avtBlueprintTreeCache.C
// ************************************************************************* //

#include "avtBlueprintTreeCache.h"

//
// //-----------------------------------------------------------------------------
// // visit includes
// //-----------------------------------------------------------------------------
#include <StringHelpers.h>
#include <TimingsManager.h>
#include <InvalidFilesException.h>
#include "FileFunctions.h"
//-----------------------------------------------------------------------------
// std lib includes
//-----------------------------------------------------------------------------
#include <map>

//-----------------------------------------------------------------------------
// conduit includes
//-----------------------------------------------------------------------------
#include "conduit.hpp"
#include "conduit_relay.hpp"
#include "conduit_relay_io_hdf5.hpp"

//-----------------------------------------------------------------------------
// mfem includes
//-----------------------------------------------------------------------------
#include "mfem.hpp"

//-----------------------------------------------------------------------------
// bp visit plugin includes
//-----------------------------------------------------------------------------
#include "avtBlueprintLogging.h"

using std::string;
using namespace conduit;
using namespace mfem;

//----------------------------------------------------------------------------/
///
/// avtBlueprintTreeCache::CacheMap Interface
///
//
//  Modifications:
//    Cyrus Harrison, Tue Nov  7 15:35:20 PST 2023
//    Add logic to limit max number of file handles held open
//
//----------------------------------------------------------------------------/
class avtBlueprintTreeCache::CacheMap
{
  public:
         CacheMap();
        ~CacheMap();

        Node    &FetchTree(int domain_id);
        Node    &FetchSidreMetaTree(int domain_id);

        hid_t    FetchHDF5Id(const std::string &file_path);

        uint64   TotalSize() const;
        uint64   TotalHDF5Ids() const;

        void     CloseHDF5FileHandle(hid_t h5_file_id);
        
        void     Release();

  private:
      std::map<int,Node>          m_nodes;
      std::map<int,Node>          m_sidre_nodes;
      int                         m_max_file_handles;
      std::map<std::string,hid_t> m_h5_ids;
      std::vector<std::string>    m_h5_path_open_order;

};

//----------------------------------------------------------------------------/
///
/// avtBlueprintTreeCache::IO Methods
///
//----------------------------------------------------------------------------/

//----------------------------------------------------------------------------/
avtBlueprintTreeCache::CacheMap::CacheMap()
: m_nodes(),
  m_sidre_nodes(),
  m_max_file_handles(256),
  m_h5_ids()
{}

//----------------------------------------------------------------------------/
avtBlueprintTreeCache::CacheMap::~CacheMap()
{
    Release();
}


//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::CacheMap::Release()
{
    // this will free all cached tree data
    m_nodes.clear();
    m_sidre_nodes.clear();
    
    // this will close all of the hdf5 file handles
    
    std::map<std::string, hid_t>::const_iterator itr;
    for(itr = m_h5_ids.begin(); itr != m_h5_ids.end(); ++itr)
    {
     
         hid_t h5_file_id = (*itr).second;
         CloseHDF5FileHandle(h5_file_id);
    }

    m_h5_ids.clear();
    m_h5_path_open_order.clear();

}

//----------------------------------------------------------------------------/
Node &
avtBlueprintTreeCache::CacheMap::FetchTree(int domain_id)
{
    return m_nodes[domain_id];
}

//----------------------------------------------------------------------------/
Node &
avtBlueprintTreeCache::CacheMap::FetchSidreMetaTree(int domain_id)
{
    return m_sidre_nodes[domain_id];
}

//----------------------------------------------------------------------------/
hid_t
avtBlueprintTreeCache::CacheMap::FetchHDF5Id(const std::string &file_path)
{
    hid_t h5_file_id = -1;

    // check if handle is open
    if ( m_h5_ids.find(file_path) == m_h5_ids.end() )
    {
        // handle is not open, check if we are at max number of handles
        if( TotalHDF5Ids() > m_max_file_handles )
        {
            // take the first entry and close it
            std::string h5_file_path_to_close = m_h5_path_open_order[0];
            hid_t h5_id_to_close = m_h5_ids[h5_file_path_to_close];
            CloseHDF5FileHandle(h5_id_to_close);
            m_h5_path_open_order.erase(m_h5_path_open_order.begin());
            m_h5_ids.erase(h5_file_path_to_close);
        }

        // assume fetch_path points to a hdf5 dataset
        // open the hdf5 file for reading
        h5_file_id = H5Fopen(file_path.c_str(),
                             H5F_ACC_RDONLY,
                             H5P_DEFAULT);
        CHECK_HDF5_ERROR(h5_file_id,
                         "Error opening HDF5 file for reading: "  << file_path);
        BP_PLUGIN_INFO("opened " << file_path << "  hdf5 id = " << h5_file_id);
        m_h5_path_open_order.push_back(file_path);
        m_h5_ids[file_path] = h5_file_id;
    }
    else
    {
        h5_file_id =  m_h5_ids[file_path];
    }

    return h5_file_id;
}

//----------------------------------------------------------------------------/
void avtBlueprintTreeCache::CacheMap::CloseHDF5FileHandle(hid_t h5_file_id)
{
    // close the hdf5 file
    CHECK_HDF5_ERROR(H5Fclose(h5_file_id),
                      "Error closing HDF5 file handle: " << h5_file_id);
}

//----------------------------------------------------------------------------/
uint64
avtBlueprintTreeCache::CacheMap::TotalSize() const
{
    uint64 res = 0;
    // count cached blueprint data
    std::map<int, Node>::const_iterator itr;
    for(itr  = m_nodes.begin(); 
        itr != m_nodes.end();
        itr++)
    {
        res += itr->second.total_bytes_allocated();
    }
    
    // also count cached sidre metadata
    for(itr  = m_sidre_nodes.begin(); 
        itr != m_sidre_nodes.end();
        itr++)
    {
        res += itr->second.total_bytes_allocated();
    }

    return res;
}

//----------------------------------------------------------------------------/
uint64
avtBlueprintTreeCache::CacheMap::TotalHDF5Ids() const
{
    return (uint64) m_h5_ids.size();
}

//----------------------------------------------------------------------------/
///
/// avtBlueprintTreeCache::IO Interface
///
//----------------------------------------------------------------------------/


class avtBlueprintTreeCache::IO
{
public:
    
    //-----------------------------------------------------------------------//
    // Generic Entry point (handles all protocols)
    //-----------------------------------------------------------------------//
    
    static void LoadBlueprintTree(avtBlueprintTreeCache &tcache,
                                  int tree_id,
                                  const std::string &protocol,
                                  const std::string &file_path,
                                  const std::string &tree_root,
                                  const std::string &tree_path,
                                  conduit::Node &out);

    //-----------------------------------------------------------------------//
    static bool ReadHDF5Slab(hid_t h5_file_id,
                             const std::string &fetch_path,
                             const DataType &dtype,
                             void *data_ptr);

private:
    //-----------------------------------------------------------------------//
    // Sidre Specific Read Helpers
    //-----------------------------------------------------------------------//

    //-----------------------------------------------------------------------//
    static void LoadSidreView(conduit::Node &sidre_meta_view,
                              avtBlueprintTreeCache &tcache,
                              int tree_id,
                              const std::string &file_path,
                              const std::string &tree_root,
                              const std::string &view_path,
                              conduit::Node &out);

    //-----------------------------------------------------------------------//
    static void LoadSidreGroup(conduit::Node &sidre_meta,
                               avtBlueprintTreeCache &tcache,
                               int tree_id,
                               const std::string &file_path,
                               const std::string &tree_root,
                               const std::string &group_path,
                               conduit::Node &out);

    //-----------------------------------------------------------------------//
    static void LoadSidreTree(conduit::Node &sidre_meta,
                              avtBlueprintTreeCache &tcache,
                              int tree_id,
                              const std::string &file_path,
                              const std::string &tree_root,
                              const std::string &tree_path,
                              const std::string &curr_path,
                              conduit::Node &out);

    //-----------------------------------------------------------------------//
    static std::string CreateSidreMetaGroupPath(const std::string &tree_path);


    //-----------------------------------------------------------------------//
    static std::string CreateSidreMetaViewPath(const std::string &tree_path);
    



};

//---------------------------------------------------------------------------//
// Helper that expands a sidre groups's meta path to the expected hdf5 path 
//---------------------------------------------------------------------------//
std::string
avtBlueprintTreeCache::IO::CreateSidreMetaGroupPath(const std::string &tree_path)
{
    // sidre/groups/ga/groups/gb/groups/gc/.../groups/gn
    
    std::ostringstream oss;
    
    std::string t_path  = tree_path;
    std::string t_curr;
    std::string t_next;

    while(t_path != "")
    {
        conduit::utils::split_path(t_path,
                                   t_curr,
                                   t_next);
                                   
        oss << "groups/" << t_curr;
        if( t_next != "")
        {
            oss << "/";
        }

        t_path = t_next;
   
    }

    return oss.str();

}

//---------------------------------------------------------------------------//
// Helper that expands a sidre view's meta path to the expected hdf5 path 
//---------------------------------------------------------------------------//
std::string
avtBlueprintTreeCache::IO::CreateSidreMetaViewPath(const std::string &tree_path)
{
    // sidre/groups/ga/groups/gb/groups/gc/.../views/gn
    
    std::ostringstream oss;
    
    std::string t_path  = tree_path;
    std::string t_curr;
    std::string t_next;

    while(t_path != "")
    {
        conduit::utils::split_path(t_path,
                                   t_curr,
                                   t_next);
        if(t_next == "")
        {
            oss << "/views/" << t_curr;
        }
        else
        {
            oss << "groups/" << t_curr << "/";
        }

        t_path = t_next;
   
    }

    return oss.str();

   
    std::string curr,next;
    
    
}



//---------------------------------------------------------------------------//
// Main Blueprint IO Load Method
//---------------------------------------------------------------------------//
void
avtBlueprintTreeCache::IO::LoadBlueprintTree(avtBlueprintTreeCache &tree_cache,
                                             int tree_id,
                                             const std::string &protocol,
                                             const std::string &file_path,
                                             const std::string &tree_root,
                                             const std::string &tree_path,
                                             Node &out)
{
    int t_load_bp_tree = visitTimer->StartTimer();
    // non sidre case
    if(protocol.find("sidre") == std::string::npos)
    {
        std::string fetch_path = tree_root + tree_path;
        BP_PLUGIN_INFO("tree cache read " 
                       << "[domain " << tree_id << "] "
                       << file_path << ":"
                       << fetch_path);
        // capture errors to allow sparsely populated trees to be read
        // e.g. a coordset, topology, or field that only exists 
        // on a subset of trees
        int t_tree_read = visitTimer->StartTimer();
        try
        {
            tree_cache.Read(file_path,fetch_path,out);
            visitTimer->StopTimer(t_tree_read, "tree read");
        }
        catch(const conduit::Error &e)
        {
            visitTimer->StopTimer(t_tree_read, "tree read");
            visitTimer->StopTimer(t_load_bp_tree, "IO::LoadBlueprintTree");
            BP_PLUGIN_EXCEPTION1(InvalidVariableException,
                                 "Failed to read tree path: "
                                 << "domain "
                                 << tree_id
                                 << " : "
                                 << fetch_path);
        }
    }
    else if( protocol == "sidre_hdf5" )
    {
        Node &sidre_meta = tree_cache.Cache().FetchSidreMetaTree(tree_id);
        
        int t_sidre_meta_read = visitTimer->StartTimer();
       
        // this path will either be a sidre group or a sidre view
        // check if either exists cached
        BP_PLUGIN_INFO("fetch sidre tree: "<< tree_path);
      
        std::string sidre_mtree_view  = CreateSidreMetaViewPath(tree_path);
        std::string sidre_mtree_group = CreateSidreMetaGroupPath(tree_path);
        
        if( !sidre_meta.has_path(sidre_mtree_group) ||
            !sidre_meta.has_path(sidre_mtree_view) )
        {
    
            // check to see if we have a group or view
            if( tree_cache.HasPath(file_path,
                                   tree_root + "/sidre/" + sidre_mtree_group) )
            {
                   // we have a group, read the meta data
                   tree_cache.Read(file_path,
                                   tree_root + "/sidre/" + sidre_mtree_group,
                                   sidre_meta[sidre_mtree_group]);

            }
            else if( tree_cache.HasPath(file_path,
                                        tree_root + "/sidre/" + sidre_mtree_view) )
            {
                   // we have a view, read the meta data
                   tree_cache.Read(file_path,
                                   tree_root + "/sidre/" + sidre_mtree_view,
                                   sidre_meta[sidre_mtree_view]);
            }
            else
            {
                visitTimer->StopTimer(t_sidre_meta_read, "fetch sidre meta data from hdf5");
                visitTimer->StopTimer(t_load_bp_tree, "IO::LoadBlueprintTree");
                BP_PLUGIN_EXCEPTION1(InvalidVariableException, 
                                     "Failed to read tree path: " 
                                     << file_path << ":"
                                     << tree_path
                                     << std::endl
                                     << "Expected to find Sidre Group: "
                                     << tree_root << "/sidre/" << sidre_mtree_group
                                     << " or "
                                     << "Sidre View: " 
                                     << tree_root << "/sidre/" << sidre_mtree_view);
            }

            visitTimer->StopTimer(t_sidre_meta_read, "fetch sidre meta data from hdf5");
        }
        else
        {
            visitTimer->StopTimer(t_sidre_meta_read, "fetch sidre meta data from cache");
        }
        
        
        int t_sidre_tree = visitTimer->StartTimer();
        
        // start a top level traversal 
        LoadSidreTree(sidre_meta,
                      tree_cache,
                      tree_id,
                      file_path,
                      tree_root,
                      tree_path,
                      "",
                      out);
        visitTimer->StopTimer(t_sidre_tree, "LoadSidreTree");
    }
    else
    {
        visitTimer->StopTimer(t_load_bp_tree, "IO::LoadBlueprintTree");
        BP_PLUGIN_EXCEPTION1(InvalidVariableException,
                             "unknown protocol" << protocol);
    }

    visitTimer->StopTimer(t_load_bp_tree, "IO::LoadBlueprintTree");
}



//---------------------------------------------------------------------------//
//---------------------------------------------------------------------------//
//
// Helpers for reading sidre style hdf5 data. 
//
//---------------------------------------------------------------------------//
//---------------------------------------------------------------------------//


//----------------------------------------------------------------------------/
// TODO CONST FOR INPUT 
void
avtBlueprintTreeCache::IO::LoadSidreView(Node &sidre_meta_view,
                                         avtBlueprintTreeCache &tree_cache,
                                         int tree_id,
                                         const std::string &file_path,
                                         const std::string &tree_root,
                                         const std::string &view_path,
                                         Node &out)
{
    // view load cases:
    //   the view is a scalar or string
    //     simply copy the "value" from the meta view
    //
    //   the view is attached to a buffer
    //     in this case we need to get the info about the buffer the view is 
    //     attached to and read the proper slab of that buffer's hdf5 dataset
    //     into a new compact node.
    //
    //   the view is has external data
    //     for this case we can follow the "tree_path" in the sidre external
    //     data tree, and fetch the hdf5 dataset that was written there.
    //
    string view_state = sidre_meta_view["state"].as_string();

    if( view_state == "STRING")
    {
        BP_PLUGIN_INFO("loading " << view_path << " as sidre string view");
        out.set(sidre_meta_view["value"]);
    }
    else if(view_state == "SCALAR")
    {
        BP_PLUGIN_INFO("loading " << view_path << " as sidre scalar view");
        out.set(sidre_meta_view["value"]);
    }
    else if( view_state == "BUFFER" )
    {
        BP_PLUGIN_INFO("loading " << view_path << " as sidre view linked to a buffer");
        // we need to fetch the buffer
        int buffer_id = sidre_meta_view["buffer_id"].to_int();

        std::ostringstream buffer_fetch_path_oss;
        buffer_fetch_path_oss << tree_root  
                       << "sidre/buffers/buffer_id_" << buffer_id;
               
        // buffer data path
        std::string buffer_data_fetch_path   = buffer_fetch_path_oss.str() + "/data";
        // we also need the buffer's schema 
        std::string buffer_schema_fetch_path = buffer_fetch_path_oss.str() + "/schema";

        Node n_buffer_schema_str;
        
        // TODO: We aren't caching this, but the final result is cached, not sure
        // we need to cache.
        
        tree_cache.Read(file_path,buffer_schema_fetch_path,n_buffer_schema_str);
       
        string buffer_schema_str = n_buffer_schema_str.as_string();
        Schema buffer_schema(buffer_schema_str);

        BP_PLUGIN_INFO("sidre buffer schema: " << buffer_schema.to_json());
        BP_PLUGIN_INFO("sidre buffer data path " << buffer_data_fetch_path);


        string view_schema_str =  sidre_meta_view["schema"].as_string();
        // create the schema we want for this view
        // it describes how the view relates to the buffer in the hdf5 file
        Schema view_schema(view_schema_str);
        BP_PLUGIN_INFO("sidre view schema: " << view_schema.to_json());
        
        // if the schema isn't compact, or if we are reading 
        // less elements than the entire buffer, 
        // we need to read a subset of the hdf5 dataset
      
        if(   !view_schema.is_compact() || 
            ( view_schema.dtype().number_of_elements() <
              buffer_schema.dtype().number_of_elements() )
          )
        {
            BP_PLUGIN_INFO("Sidre View from Buffer Slab Fetch Case");
            //
            // Create a compact schema to describe our desired output data
            //
            Schema view_schema_compact;
            view_schema.compact_to(view_schema_compact);
            // setup and allocate the output node
            out.set(view_schema_compact);

            // ---------------------------------------------------------------
            // BUFFER-SLAB FETCH
            // ---------------------------------------------------------------
            //
            // we can use hdf5 slab fetch if the the dtype.id() of the buffer
            // and the view are the same. 
            // 
            //  otherwise, we will have to fetch the entire buffer since
            //  hdf5 doesn't support byte level striding.

            if(
                tree_cache.Read(file_path,
                                buffer_data_fetch_path,
                                view_schema.dtype(),
                                out)
                )
            {
                BP_PLUGIN_INFO("Sidre View from Buffer Slab Fetch Case Failed");
                // ---------------------------------------------------------------
                // Fall back to Non BUFFER-SLAB FETCH
                // ---------------------------------------------------------------
                // this reads the entire buffer to get the proper subset
                Node n_buff;
                Node n_view;

                tree_cache.Read(file_path,
                                buffer_data_fetch_path,
                                n_buff);
                // create our view on the buffer
                n_view.set_external(view_schema,n_buff.data_ptr());
                // compact the view to our output
                n_view.compact_to(out);
            }
            else
            {
                BP_PLUGIN_INFO("Sidre View from Buffer Slab Fetch Case Successful");
            }
        }
        else
        {
            
            tree_cache.Read(file_path,
                            buffer_data_fetch_path,
                            out);
        }
    }
    else if( view_state == "EXTERNAL" )
    {
        BP_PLUGIN_INFO("loading " << view_path << " as sidre external view");

        std::string fetch_path = tree_root + "sidre/external/" + view_path;

        BP_PLUGIN_INFO("relay:io::hdf5_read " 
                       << "[domain " << tree_id << "] " 
                       << file_path  << ":"
                       << fetch_path);
        
        tree_cache.Read(file_path,
                        fetch_path,
                        out);

    }
    else
    {
        BP_PLUGIN_EXCEPTION1( InvalidVariableException,
                              "unsupported sidre view state: " << view_state );
    }
}


//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::IO::LoadSidreGroup(Node &sidre_meta,
                                          avtBlueprintTreeCache &tree_cache,
                                          int tree_id,
                                          const std::string &file_path,
                                          const std::string &tree_root,
                                          const std::string &group_path,
                                          Node &out)
{
    // load this group's children groups and views
    NodeIterator g_itr = sidre_meta["groups"].children();
    while(g_itr.has_next())
    {
        Node &g = g_itr.next();
        string g_name = g_itr.name();
        BP_PLUGIN_INFO("loading " << group_path << g_name << " as group");
        std::string cld_path = group_path + g_name;
        LoadSidreGroup(g,
                       tree_cache,
                       tree_id,
                       file_path,
                       tree_root,
                       cld_path + "/",
                       out[g_name]);
    }    
    
    NodeIterator v_itr = sidre_meta["views"].children();
    while(v_itr.has_next())
    {
        Node &v = v_itr.next();
        string v_name = v_itr.name();
        BP_PLUGIN_INFO("loading " << group_path << v_name << " as view");
        std::string cld_path = group_path + v_name;
        LoadSidreView(v,
                      tree_cache,
                      tree_id,
                      file_path,
                      tree_root,
                      cld_path,
                      out[v_name]);
        
    }
}


//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::IO::LoadSidreTree(Node &sidre_meta,
                                         avtBlueprintTreeCache &tree_cache,
                                         int tree_id,
                                         const std::string &file_path,
                                         const std::string &tree_root,
                                         const std::string &tree_path,
                                         const std::string &curr_path,
                                         Node &out)
{
    // we want to pull out a sub-tree of the sidre group hierarchy 
    //
    // descend down to "tree_path" in sidre meta
    
    string tree_curr;
    string tree_next;
    conduit::utils::split_path(tree_path,tree_curr,tree_next);
    
    if( sidre_meta["groups"].has_path(tree_curr) )
    {
        BP_PLUGIN_INFO(curr_path << tree_curr << " is a group");
        if(tree_next.size() == 0)
        {
            LoadSidreGroup(sidre_meta["groups"][tree_curr],
                           tree_cache,
                           tree_id,
                           file_path,
                           tree_root,
                           curr_path + tree_curr  + "/",
                           out);
        }
        else // keep descending 
        {
            LoadSidreTree(sidre_meta["groups"][tree_curr],
                          tree_cache,
                          tree_id,
                          file_path,
                          tree_root,
                          tree_next,
                          curr_path + tree_curr  + "/",
                          out);
        }
    }
    else if( sidre_meta["view"].has_path(tree_curr) )
    {
        BP_PLUGIN_INFO(curr_path << tree_curr << " is a view");
        if(tree_next.size() != 0)
        {
            BP_PLUGIN_EXCEPTION1( InvalidVariableException,
                                  "path extends beyond sidre view (views are leaves)");
        }
        else
        {
            LoadSidreView(sidre_meta["view"][tree_curr],
                          tree_cache,
                          tree_id,
                          file_path,
                          tree_root,
                          curr_path + tree_curr  + "/",
                          out);
        }
    }
    else
    {
        BP_PLUGIN_EXCEPTION1( InvalidVariableException,
                              "sidre tree path " << tree_curr << " does not exist");
    }
}

//---------------------------------------------------------------------------//
//---------------------------------------------------------------------------//
//
// Helper for reading HDF5 data 
// (for cases beyond what conduit::relay provides)
//
//---------------------------------------------------------------------------//
//---------------------------------------------------------------------------//


// ****************************************************************************
bool
avtBlueprintTreeCache::IO::ReadHDF5Slab(hid_t h5_file_id,
                                        const std::string &fetch_path,
                                        const DataType &dtype,
                                        void *data_ptr)
{
    // open the dataset
    hid_t h5_dset_id = H5Dopen( h5_file_id, fetch_path.c_str(),H5P_DEFAULT);

    CHECK_HDF5_ERROR(h5_dset_id,
                     "Error opening HDF5 dataset at: " << fetch_path);


    // get info about the dataset
    hid_t h5_dspace_id = H5Dget_space(h5_dset_id);
    CHECK_HDF5_ERROR(h5_dspace_id,
                     "Error reading HDF5 Dataspace: " << h5_dset_id);

    // check for empty case
    if(H5Sget_simple_extent_type(h5_dspace_id) == H5S_NULL)
    {
        // we have an error, but to try to clean up the hdf5 handles
        // before we issue the error.
        
        CHECK_HDF5_ERROR(H5Sclose(h5_dspace_id),
                          "Error closing HDF5 data space: " << fetch_path);

        CHECK_HDF5_ERROR(H5Dclose(h5_dset_id),
                          "Error closing HDF5 dataset: " << fetch_path);
        BP_PLUGIN_EXCEPTION1( InvalidVariableException,
                              "Can't slab fetch from an empty hdf5 data set.");
    }

    hid_t h5_dtype_id  = H5Dget_type(h5_dset_id);

    CHECK_HDF5_ERROR(h5_dtype_id,
                     "Error reading HDF5 Datatype: "
                     << h5_dset_id);

    // ounds check  (check that we are fetching a subset of the elems)
    index_t  h5_nelems = H5Sget_simple_extent_npoints(h5_dspace_id);
    if( dtype.number_of_elements() > h5_nelems)
    {
        // we have an error, but to try to clean up the hdf5 handles
        // before we issue the error.
        
        CHECK_HDF5_ERROR(H5Sclose(h5_dspace_id),
                          "Error closing HDF5 data space: " << fetch_path);

        CHECK_HDF5_ERROR(H5Dclose(h5_dset_id),
                          "Error closing HDF5 dataset: " << fetch_path);

        BP_PLUGIN_EXCEPTION1( InvalidVariableException,
                              "Can't slab fetch a buffer larger than the source"
                              " hdf5 data set. Requested number of elements" 
                              << dtype.number_of_elements()
                              << " hdf5 dataset number of elements" << h5_nelems);
    }


    // we need to compute an offset, stride, and element bytes
    // that will work for reading in the general case
    // right now we assume the dest type of data and the hdf5 datasets
    // data type are compatible  
    
    // conduit's offsets, strides, are all in terms of bytes
    // hdf5's are in terms of elements
    
    // what we really want is a way to read bytes from the hdf5 dset with
    // out any type conversion, but that doesn't exist.

    // general support would include reading a a view of one type that
    //  points to a buffer of another
    // (for example a view of doubles that is defined on a buffer of bytes)

    // but hdf5 doens't support slab fetch across datatypes
    // so for now we make sure the datatype is consistent. 

    // TODO: Note we now have access the buffer datatype outside of this
    //  method, so we may be able to refactor at a higher level to catch
    // this case.  
    
    DataType h5_dt = conduit::relay::io::hdf5_dtype_to_conduit_dtype(h5_dtype_id,1);
    
    if( h5_dt.id() != dtype.id() )
    {
        // cleanup hdf5 handles 
        
        // close the dspace handle
        CHECK_HDF5_ERROR(H5Sclose(h5_dspace_id),
                          "Error closing HDF5 data space: " << fetch_path);

        // close the dataset
        CHECK_HDF5_ERROR(H5Dclose(h5_dset_id),
                           "Error closing HDF5 dataset: " << fetch_path);
        
        // close the hdf5 file
        CHECK_HDF5_ERROR(H5Fclose(h5_file_id),
                         "Error closing HDF5 file: " << fetch_path);
                         
        BP_PLUGIN_INFO("Cannot fetch hdf5 slab of buffer and view are"
                        "different data types.")
        return false;
    }
    
    /// adj endianness if necessary 

    if(!h5_dt.endianness_matches_machine())
    {
        // if they don't match, modify the dt
        // and get the proper hdf5 data type handle
        h5_dt.set_endianness(Endianness::machine_default());

        // clean up our old handle
        CHECK_HDF5_ERROR(H5Tclose(h5_dtype_id),
                         "Error closing HDF5 Datatype: " << h5_dtype_id);

        // get ref to standard variant of this dtype
        h5_dtype_id  = conduit::relay::io::conduit_dtype_to_hdf5_dtype(h5_dt);

        CHECK_HDF5_ERROR(h5_dtype_id,
                         "Error creating HDF5 Datatype");

        // copy this handle, b/c clean up code later will close it
        h5_dtype_id  = H5Tcopy(h5_dtype_id);
        CHECK_HDF5_ERROR(h5_dtype_id,
                         "Error copying HDF5 Datatype: " << h5_dtype_id);
    }

    hid_t h5_status    = 0;

    hsize_t elem_bytes = dtype.element_bytes();
    hsize_t offset  = dtype.offset() / elem_bytes; // in bytes, convert to elems
    hsize_t stride  = dtype.stride() / elem_bytes; // in bytes, convert to elems
    hsize_t num_ele = dtype.number_of_elements();
    
    BP_PLUGIN_INFO("slab dtype: " << dtype.to_json());
    
    BP_PLUGIN_INFO("hdf5 slab: "  <<
                   " element_offset: " << offset <<
                   " element_stride: " << stride <<
                   " number_of_elements: " << num_ele);
    
    h5_status = H5Sselect_hyperslab(h5_dspace_id,
                                    H5S_SELECT_SET,
                                    &offset,
                                    &stride,
                                    &num_ele,
                                    0); // 0 here means NULL pointers; HDF5 *knows* dimension is 1
    // check subset sel
    CHECK_HDF5_ERROR(h5_status,
                      "Error selecting hyper slab from HDF5 dataspace: " << h5_dspace_id);

    hid_t h5_dspace_compact_id = H5Screate_simple(1,
                                                  &num_ele,
                                                  NULL);

    CHECK_HDF5_ERROR(h5_dspace_id,"Failed to create HDF5 Dataspace");

    h5_status = H5Dread(h5_dset_id, // data set id
                        h5_dtype_id, // memory type id  // use same data type
                        h5_dspace_compact_id,  // memory space id ...
                        h5_dspace_id, // file space id
                        H5P_DEFAULT,
                        data_ptr);

    // check read
    CHECK_HDF5_ERROR(h5_status,
                      "Error reading bytes from HDF5 dataset: " << h5_dset_id);

    // close the data space 
    CHECK_HDF5_ERROR(H5Sclose(h5_dspace_id),
                      "Error closing HDF5 data space: " << fetch_path);

    // close the compact data space 
    CHECK_HDF5_ERROR(H5Sclose(h5_dspace_compact_id),
                      "Error closing HDF5 compact memory data space" << fetch_path);

    // close the data type
    CHECK_HDF5_ERROR(H5Tclose(h5_dtype_id),
                     "Error closing HDF5 Datatype: " << h5_dtype_id);

    // close the dataset
    CHECK_HDF5_ERROR(H5Dclose(h5_dset_id),
                      "Error closing HDF5 dataset: " << fetch_path);

    return true;
}


//----------------------------------------------------------------------------/
///
/// avtBlueprintTreeCache Methods
///
//----------------------------------------------------------------------------/


//----------------------------------------------------------------------------/
avtBlueprintTreeCache::avtBlueprintTreeCache()
: m_file_pattern(""),
  m_tree_pattern(""),
  m_num_files(0),
  m_num_trees(0),
  m_protocol("hdf5"),
  m_cache_map(NULL)
{
    m_cache_map = new CacheMap();
    
}

//----------------------------------------------------------------------------/
avtBlueprintTreeCache::~avtBlueprintTreeCache()
{
    delete m_cache_map;
}

//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::Release()
{
    m_cache_map->Release();
}

//----------------------------------------------------------------------------/
std::string
avtBlueprintTreeCache::Expand(const std::string pattern,
                              int idx) const
{
    const std::vector<std::string> patterns{"%08d", "%07d", "%06d", "%05d",
                                            "%04d", "%03d", "%02d", "%d"};
    for(const auto &p : patterns)
    {
        if(pattern.find(p) != std::string::npos)
        {
            char buff[16];
            snprintf(buff, 16, p.c_str(), idx);
            return StringHelpers::Replace(pattern,
                                          p,
                                          std::string(buff));
        }
    }
    if(pattern.find("%") != std::string::npos)
    {
        BP_PLUGIN_EXCEPTION1(InvalidFilesException,
            "Unsupported pattern in \"" << pattern << "\".");
    }
    return pattern;
}


//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::SetRootDir(const std::string &root_dir)
{
    m_root_dir = root_dir;
}

//----------------------------------------------------------------------------/
void
avtBlueprintTreeCache::SetFilePattern(const std::string &file_pattern)
{
    m_file_pattern = file_pattern;
}

void
avtBlueprintTreeCache::SetTreePattern(const std::string &tree_pattern)
{
    m_tree_pattern = tree_pattern;
}

//-------------------------------------------------------------------//
void
avtBlueprintTreeCache::SetNumberOfFiles(int num_files)
{
    m_num_files = num_files;
}

//-------------------------------------------------------------------//
void
avtBlueprintTreeCache::SetNumberOfTrees(int num_trees)
{
    m_num_trees = num_trees;
}

//-------------------------------------------------------------------//
void
avtBlueprintTreeCache::SetProtocol(const std::string &protocol)
{
    m_protocol = protocol;
}

//-------------------------------------------------------------------//
void
avtBlueprintTreeCache::AddMeshParitionMap(const std::string   &mesh_name,
                                          const std::string   &partition_pattern)
{
    m_part_maps[mesh_name]["pattern"] = partition_pattern;
}


//-------------------------------------------------------------------//
void
avtBlueprintTreeCache::AddMeshParitionMap(const std::string   &mesh_name,
                                          const std::string   &partition_pattern,
                                          const conduit::Node &partition_map)
{
    Node &part_entry =  m_part_maps[mesh_name];
    part_entry["pattern"] = partition_pattern;
    part_entry["map"]     = partition_map;

    // find max possible domain id
    index_t_accessor doms = partition_map["domain"].value();
    index_t num_domains   = doms.max() + 1;

    // fill out domain to tree map
    part_entry["domain_to_tree"].set(DataType::index_t(num_domains));
    index_t_array dom_to_tree_vals = part_entry["domain_to_tree"].value();
    for(index_t i=0;i<num_domains;i++)
    {
        dom_to_tree_vals[doms[i]] = i;
    }
}




//-----------------------------------------------------------------------------
// This uses a mapping scheme created by ascent + conduit
// Note: We will support explicit maps from the bp index in the future.
// TODO: We should cache this, not regen every fetch
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
static void
gen_domain_to_file_map(int num_domains,
                       int num_files,
                       Node &out)
{
    int num_domains_per_file = num_domains / num_files;
    int left_overs = num_domains % num_files;

    out["global_domains_per_file"].set(DataType::int32(num_files));
    out["global_domain_offsets"].set(DataType::int32(num_files));
    out["global_domain_to_file"].set(DataType::int32(num_domains));
    
    int32_array v_domains_per_file = out["global_domains_per_file"].value();
    int32_array v_domains_offsets  = out["global_domain_offsets"].value();
    int32_array v_domain_to_file   = out["global_domain_to_file"].value();
    
    // setup domains per file
    for(int f=0; f < num_files; f++)
    {
        v_domains_per_file[f] = num_domains_per_file;
        if( f < left_overs)
            v_domains_per_file[f]+=1;
    }

    // prefix sum to calc offsets
    for(int f=0; f < num_files; f++)
    {
        v_domains_offsets[f] = v_domains_per_file[f];
        if(f > 0)
            v_domains_offsets[f] += v_domains_offsets[f-1];
    }
    
    // do assignment, create simple map
    int f_idx = 0;
    for(int d=0; d < num_domains; d++)
    {
        if(d >= v_domains_offsets[f_idx])
            f_idx++;
        v_domain_to_file[d] = f_idx;
    }
}

//-------------------------------------------------------------------//
std::string
avtBlueprintTreeCache::GenerateFilePath(const std::string &mesh_name,
                                        int tree_id) const
{
    std::string res;
    // check for the partition map case
    if(m_part_maps.has_child(mesh_name)) 
    {
        BP_PLUGIN_INFO( "GenerateFilePath using partition map for mesh: "
                        << mesh_name);
        // split out file
        std::string tmp;
        utils::split_string(GeneratePartitionMapFullPath(m_part_maps[mesh_name],tree_id),
                            ":/",
                            res,  // result is before sep
                            tmp);
    }
    else // legacy case
    {
        int file_id = -1;

        if(m_num_trees == m_num_files)
        {
            file_id = tree_id;
        }
        else if(m_num_files == 1)
        {
            file_id = 0;
        }
        else
        {
            Node d2f_map;
            gen_domain_to_file_map(m_num_trees,
                                    m_num_files,
                                    d2f_map);
            int num_domains_per_file = m_num_trees / m_num_files;
            int left_overs = m_num_trees % m_num_files;
            int32_array v_domain_to_file = d2f_map["global_domain_to_file"].value();
            file_id = v_domain_to_file[tree_id];
        }

        BP_PLUGIN_INFO( "tree_id: " << tree_id 
                        << " ==> file_id: " << file_id);

        res = Expand(m_file_pattern,file_id);
    }
    return res;
}

//-------------------------------------------------------------------//
std::string
avtBlueprintTreeCache::GenerateTreePath(const std::string &mesh_name,
                                        int tree_id) const
{
    std::string res;
    // check for the partition map case
    if(m_part_maps.has_child(mesh_name)) 
    {
        BP_PLUGIN_INFO( "GenerateTreePath using partition map for mesh: "
                        << mesh_name);
        // split out sub path
        std::string tmp;
        utils::split_string(GeneratePartitionMapFullPath(m_part_maps[mesh_name],tree_id),
                            ":/",
                            tmp,
                            res); // result is before sep
        // tree path should always end with "/"
        if( (res.size() > 0) && (res[res.size()-1] != '/') )
        {
            res += "/";
        }
    }
    else // legacy case
    {
        // the tree path should always end in a /
        res = Expand(m_tree_pattern,tree_id);
        if( (res.size() > 0) && (res[res.size()-1] != '/') )
        {
            res += "/";
        }
    }
    return res;
}

//-------------------------------------------------------------------//
std::string
avtBlueprintTreeCache::GeneratePartitionMapFullPath(const conduit::Node &mesh_part_info,
                                                    int tree_id) const
{
    // general case
    if(mesh_part_info.has_child("map"))
    {
        std::string full_pattern = mesh_part_info["pattern"].as_string();
        if(full_pattern[0] !=  VISIT_SLASH_STRING[0])
        {
            full_pattern  = m_root_dir + string(VISIT_SLASH_STRING) + full_pattern;
        }
        
        index_t_array dom_to_tree_vals = mesh_part_info["domain_to_tree"].value();
        index_t dom_lookup = dom_to_tree_vals[tree_id];
        return conduit::utils::format(full_pattern,
                                      mesh_part_info["map"],
                                      dom_lookup);
    }
    else  // part_pattern w/o any map == single file case.
    {
        std::string full_pattern = mesh_part_info["pattern"].as_string();
        if(full_pattern[0] !=  VISIT_SLASH_STRING[0])
        {
            full_pattern  = m_root_dir + string(VISIT_SLASH_STRING) + full_pattern;
        }
        return full_pattern;
    }
}


//---------------------------------------------------------------------------//
void
avtBlueprintTreeCache::FetchBlueprintTree(int tree_id, 
                                          const std::string &mesh_name,
                                          const std::string &sub_tree_path,
                                          conduit::Node &out)
{
    int t_fetch_tree = visitTimer->StartTimer();
    
    Node &cache = m_cache_map->FetchTree(tree_id);

    if( cache.has_path(sub_tree_path) )
    {
        out.set_external(cache[sub_tree_path]);
    }
    else
    {
        Node &cache_sub_tree = cache[sub_tree_path];
        IO::LoadBlueprintTree(*this,
                              tree_id,
                              m_protocol,
                              GenerateFilePath(mesh_name,tree_id),
                              GenerateTreePath(mesh_name,tree_id),
                              sub_tree_path,
                              cache_sub_tree);

        out.set_external(cache_sub_tree);
    }

    BP_PLUGIN_INFO("Blueprint Tree Cache Size (Post fetch of"
                   << " domain: " << tree_id
                   << " sub path: " << sub_tree_path << ") = "
                   << m_cache_map->TotalSize() << " bytes");
    visitTimer->StopTimer(t_fetch_tree, "FetchBlueprintTree");

}


//---------------------------------------------------------------------------//
avtBlueprintTreeCache::CacheMap &
avtBlueprintTreeCache::Cache()
{
    return *m_cache_map;
}


//-----------------------------------------------------------------------//
void
avtBlueprintTreeCache::Read(const std::string &file_path,
                            const std::string &fetch_path,
                            conduit::Node &out)
{
    // switch on protocol
    if(m_protocol.find("hdf5") != std::string::npos)
    {
        hid_t h5_file_id = m_cache_map->FetchHDF5Id(file_path);
        conduit::relay::io::hdf5_read(h5_file_id,fetch_path,out);
    }
    else
    {
        // TODO proper sub tree, or read and cache at higher lvl
        Node n_full;
        conduit::relay::io::load(file_path,m_protocol,n_full);
        out = n_full[fetch_path];
    }
}


    
//-----------------------------------------------------------------------//
// Note: Only used by sidre_hdf5
bool
avtBlueprintTreeCache::HasPath(const std::string &file_path,
                               const std::string &fetch_path)
{
    hid_t h5_file_id = m_cache_map->FetchHDF5Id(file_path);
    return conduit::relay::io::hdf5_has_path(h5_file_id,fetch_path);
}

//-----------------------------------------------------------------------//
// Note: Only used by sidre_hdf5
bool
avtBlueprintTreeCache::Read(const std::string &file_path,
                            const std::string &fetch_path,
                            const conduit::DataType &dtype,
                            conduit::Node &out)
{
    hid_t h5_file_id = m_cache_map->FetchHDF5Id(file_path);

    void *data_ptr = out.data_ptr();
    bool res = IO::ReadHDF5Slab(h5_file_id,
                                fetch_path,
                                dtype,
                                data_ptr);

    return res;
}


